# Regress essay content on SAT score for Tables 3 and S2

library(caret)
library(tidyverse)

merged_df <- read.csv("merged_final.csv")

set.seed(1993)
merged_folds <- trainControl(method = "cv", number = 10)
merged_topics <- as.matrix(log(merged_df[,6:75]))

min_merged_cor <- abs(cor(merged_df$RSAT_TOTAL_SCORE, 
                              log(merged_df[,6:75])))
# Drop min cor, topic 16

merged_topics <- merged_topics[,-16]

merged_df <- as.data.frame(cbind(merged_df$RSAT_TOTAL_SCORE, merged_topics))

merged_mod <- train(V1 ~ ., method = "lm",
                    data = merged_df, trControl = merged_folds)

print(merged_mod)


################################################################################

creativ_df <- read.csv("creative_final.csv")

set.seed(1993)
creativ_folds <- trainControl(method = "cv", number = 10)
creativ_topics <- as.matrix(log(creativ_df[,6:55]))

min_creativ_cor <- abs(cor(creativ_df$RSAT_TOTAL_SCORE, 
                          log(creativ_df[,6:55])))
# Drop min cor, topic 49

creativ_topics <- creativ_topics[,-49]

creativ_df <- as.data.frame(cbind(creativ_df$RSAT_TOTAL_SCORE, creativ_topics))

creativ_mod <- train(V1 ~ ., method = "lm",
                    data = creativ_df, trControl = creativ_folds)

print(creativ_mod)

################################################################################

signif_df <- read.csv("signif_final.csv")

set.seed(1993)
signif_folds <- trainControl(method = "cv", number = 10)
signif_topics <- as.matrix(log(signif_df[,6:55]))

min_signif_cor <- abs(cor(signif_df$RSAT_TOTAL_SCORE, 
                           log(signif_df[,6:55])))
# Drop min cor, topic 40

signif_topics <- signif_topics[,-40]

signif_df <- as.data.frame(cbind(signif_df$RSAT_TOTAL_SCORE, signif_topics))

signif_mod <- train(V1 ~ ., method = "lm",
                     data = signif_df, trControl = signif_folds)

print(signif_mod)
